package com.example.crawler.controller;

import com.alibaba.excel.util.CollectionUtils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.example.crawler.mapper.CrawlerAllMapper;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;

import javax.annotation.Resource;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

@Controller
@RequestMapping("/schedule/")
//@Component
public class CrawlerAllPropSchedule {

    private static Logger logger = LoggerFactory.getLogger(CrawlerAllPropSchedule.class);

    @Resource
    private CrawlerAllMapper crawlerAllMapper;


    /**
     * 组织数据汇总
     */
//    @XxlJob("orgDataSchedule")
    @GetMapping(value = "/getComList")
    public void orgDataSchedule() throws Exception {
//        XxlJobHelper.log("XXL-JOB, orgDataSchedule.");
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        try {
            long start = System.currentTimeMillis();
            logger.info(String.format("开始爬取拟上市数据: %s", start));
            Calendar calendar = Calendar.getInstance();
            for (int i = 1; i <= 365; i++) {
                calendar.set(Calendar.HOUR_OF_DAY, -24);
                Date date = calendar.getTime();
                // 1、爬取沪深数据
                crawlerHsProps(date);
                logger.info("---- 1. 沪深数据爬取完成！" + sdf.format(date));

                // 2、爬取北交所数据
                crawlerBJProps(date);
                logger.info("---- 2. 北交所数据爬取完成！");
            }
            // 3、香港交易所数据
            crawlerHkexnewsData();
            logger.info("---- 3. 香港交易所数据爬取完成！");

//            // 4、纽约交易所数据
            crawlerNasdaqData();
            logger.info("---- 4. 纽交所数据爬取完成！");

            logger.info(String.format("爬取拟上市数据完成耗时: %s", System.currentTimeMillis() - start));
        } catch (Exception e) {
            logger.error(e.getMessage());
        }
    }


    private void crawlerNasdaqData() throws IOException, ParseException {
        List<Map<String, String>> companyList = new ArrayList<>();
        Calendar calendar = Calendar.getInstance();
        int year = calendar.get(Calendar.YEAR);
        int month = calendar.get(Calendar.MONTH) + 1;
        // 网址
        List<String> urlList = new ArrayList<>();
//      String url = "https://api.nasdaq.com/api/ipo/calendar?date=2022-08"; // 纳斯达克
        String url = "https://api.nasdaq.com/api/ipo/calendar?date=" + year + "-" + (month > 9 ? month : "0" + month); // 纳斯达克
        urlList.add(url);

        // 爬取数据
        for (String urlStr : urlList) {
            System.out.println(urlStr);
            companyList.addAll(crawlerData(urlStr, "nasdaq"));
        }

        for (Map<String, String> comMap : companyList) {
            if (CollectionUtils.isEmpty(crawlerAllMapper.queryByName(comMap.get("cname")))) {
                crawlerAllMapper.insert(comMap);
            } else {
                crawlerAllMapper.deleteByname(comMap.get("cname"));
                crawlerAllMapper.insert(comMap);
            }
        }

        System.out.println("OK");
    }

    private void crawlerHkexnewsData() throws IOException {
        List<Map<String, String>> companyList = new ArrayList<>();
        // 网址
        String url1 = "https://www1.hkexnews.hk/ncms/json/eds/appactive_appphip_sehk_c.json?_=1661822681203"; // 披露易 MAIN BOARD
        String url2 = "https://www1.hkexnews.hk/ncms/json/eds/appactive_app_gem_e.json?_=1671506643848";  // 披露易 GEM
        List<String> urlList = new ArrayList<>();
        urlList.add(url1);
        urlList.add(url2);

        // 爬取数据
        for (String urlStr : urlList) {
            companyList.addAll(crawlerData(urlStr, "hkexnews"));
        }
        for (Map<String, String> comMap : companyList) {
            if (CollectionUtils.isEmpty(crawlerAllMapper.queryByName(comMap.get("cname")))) {
                crawlerAllMapper.insert(comMap);
            } else {
                crawlerAllMapper.deleteByname(comMap.get("cname"));
                crawlerAllMapper.insert(comMap);
            }
        }
        System.out.println("OK");
    }


    private List crawlerData(String url, String functionType) throws IOException {
        List companyList = new ArrayList<>();
        //创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //创建HttpGet请求
        HttpGet httpGet = new HttpGet(url);
        httpGet.setHeader("User-Agent", "PostmanRuntime/7.29.2");

        //设置请求参数
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(100000)            // 设置创建连接的最长时间
                .setConnectionRequestTimeout(50000)   // 设置获取连接的最长时间
                .setSocketTimeout(10 * 100000)        // 设置数据传输的最长时间
                .build();

        httpGet.setConfig(requestConfig);

        CloseableHttpResponse response = null;
        try {
            //使用HttpClient发起请求
            response = httpClient.execute(httpGet);

            //判断响应状态码是否为200
            if (response.getStatusLine().getStatusCode() == 200) {
                //如果为200表示请求成功，获取返回数据
                String content = EntityUtils.toString(response.getEntity(), "UTF-8");
                //打印数据
                HashMap hashMap = JSON.parseObject(content, HashMap.class);

                switch (functionType) {
                    case "nasdaq":
                        companyList = getNasdqResultList(hashMap);
                        break;
                    case "hkexnews":
                        getHkResultList(companyList, hashMap);
                        break;
                    default:
                        break;

                }
            }

        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            //释放连接
            if (response == null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                httpClient.close();
            }
        }
        return companyList;
    }

    public void getHkResultList(List<Map<String, Object>> companyList, HashMap hashMap) {
        JSONArray appList = JSONArray.parseArray(hashMap.get("app").toString());
        for (Object obj : appList) {
            Map map = JSONObject.parseObject(JSONObject.toJSONString(obj), Map.class);
            Map<String, Object> company = new HashMap<>();
            String dateStr = getDateStrForG(String.valueOf(map.get("d")));
            company.put("shijian", dateStr);
            String contentStr = String.valueOf(map.get("a"));
            company.put("cname", contentStr);
            company.put("jys", "香港交易所");
            companyList.add(company);
        }
    }

    private String getDateStrForG(String dateStr) {
        return dateStr.substring(6, 10) + "-"
                + dateStr.substring(3, 5) + "-"
                + dateStr.substring(0, 2);
    }

    public List<Map<String, Object>> getNasdqResultList(HashMap hashMap) {
        // 拟上市企业 = “Filings”申请 - “Priced”定价 - “Withdrawn”撤回
        Map data = JSONObject.parseObject(String.valueOf(hashMap.get("data")), Map.class);
        List<Map<String, Object>> companyList = new ArrayList<>();
        if (data != null) {
            Map priced = JSONObject.parseObject(String.valueOf(data.get("priced")), Map.class);
            Map upcoming = JSONObject.parseObject(String.valueOf(data.get("upcoming")), Map.class);
            Map filed = JSONObject.parseObject(String.valueOf(data.get("filed")), Map.class);
            Map withdrawn = JSONObject.parseObject(String.valueOf(data.get("withdrawn")), Map.class);
            List<Map<String, Object>> pricedList = extracted(priced, "priced");
            List<Map<String, Object>> upcomingList = extracted(upcoming, "upcoming");
            List<Map<String, Object>> filedList = extracted(filed, "filed");
            List<Map<String, Object>> withdrawnList = extracted(withdrawn, "withdrawn");
            companyList.addAll(filedList);
            companyList.removeAll(pricedList);
            companyList.removeAll(upcomingList);
            companyList.removeAll(withdrawnList);
        }
        return companyList;
    }


    private List<Map<String, Object>> extracted(Map<String, Object> objectMap, String flag) {
        List<Map<String, Object>> otherList = new ArrayList<>();
        if (objectMap.get("rows") != null) {
            JSONArray rows = JSONArray.parseArray(String.valueOf(objectMap.get("rows")));
            for (Object obj : rows) {
                Map<String, Object> company = new HashMap<>();
                Map map = JSONObject.parseObject(JSONObject.toJSONString(obj), Map.class);
                company.put("stock_code", map.get("proposedTickerSymbol") == null ? null : String.valueOf(map.get("proposedTickerSymbol")));
                company.put("cname", String.valueOf(map.get("companyName")));

                switch (flag) {
                    case "priced":      // 定价
                        company.put("shijian", getDateStr(String.valueOf(map.get("pricedDate"))));
                        company.put("jys", "纳斯达克");
                        otherList.add(company);
                        break;
                    case "upcoming":    // 即将上市
                        company.put("shijian", getDateStr(String.valueOf(map.get("expectedPriceDate"))));
                        company.put("jys", "纳斯达克");
                        otherList.add(company);
                        break;
                    case "withdrawn":       // 撤回
                    case "filed":       // 申请
                        company.put("shijian", getDateStr(String.valueOf(map.get("filedDate"))));
                        company.put("jys", "纳斯达克");
                        otherList.add(company);
                        break;
                    default:
                        break;
                }

            }
        }
        return otherList;
    }

    private String getDateStr(String dateStr) {
        return dateStr.substring(6, 10) + "-"
                + dateStr.substring(0, 2) + "-"
                + dateStr.substring(3, 5);
    }

    private void crawlerBJProps(Date date) {
        String totalPages = new String();
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");//设置日期格式
        HttpClient httpClient = new HttpClient();
        CloseableHttpResponse response = null;
        PostMethod postMethod = new PostMethod("http://www.bse.cn/disclosureInfoController/zoneInfoResult.do?callback=jQuery331_1668567064387");
        postMethod.setRequestHeader("User-Agent", "PostmanRuntime/7.29.2");
        NameValuePair[] data = {
                new NameValuePair("page", "0"),
                new NameValuePair("disclosureType", "9530"),
                new NameValuePair("startTime", df.format(date)),
                new NameValuePair("endTime", df.format(date))
        };
        postMethod.setRequestBody(data);
        try {
            //使用HttpClient发起请求
            httpClient.executeMethod(postMethod); // 执行POST方法
            String result = postMethod.getResponseBodyAsString();
            //打印数据
            System.out.println(result.substring(24, result.length() - 1));
            if (!StringUtils.isEmpty(result.substring(24, result.length() - 1))) {
                List<Map> maps = JSONArray.parseArray(result.substring(24, result.length() - 1), Map.class);
                totalPages = JSON.parseObject(maps.get(0).get("listInfo").toString(), Map.class).get("totalPages").toString();
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            //释放连接
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

        for (int i = 0; i <= Integer.parseInt(totalPages); i++) {
            System.out.println(i);
            execute1(i, date);
        }
        System.out.println("执行结束！！！！！！！！！");
    }

    private void execute1(int i, Date date) {
        // 网址
        String url = "http://www.bse.cn/disclosureInfoController/zoneInfoResult.do?callback=jQuery331_1668567064387";
//        String url = "http://www.bse.cn/projectNewsController/infoResult.do?callback=jQuery331_1669962332639";
        HttpClient httpClient = new HttpClient();

        //创建HttpGet请求
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");//设置日期格式
        PostMethod postMethod = null;
        postMethod = new PostMethod(url);
        postMethod.setRequestHeader("User-Agent", "PostmanRuntime/7.29.2");
        NameValuePair[] data = {
                new NameValuePair("page", String.valueOf(i)),
                new NameValuePair("disclosureType", "9530"),
                new NameValuePair("startTime", df.format(date)),
                new NameValuePair("endTime", df.format(date))
        };
        postMethod.setRequestBody(data);

        CloseableHttpResponse response = null;
        execute2(httpClient, postMethod, response);
    }

    private void execute2(HttpClient httpClient, PostMethod postMethod, CloseableHttpResponse response) {
        try {
            //使用HttpClient发起请求
            httpClient.executeMethod(postMethod); // 执行POST方法
            String result = postMethod.getResponseBodyAsString();
            //打印数据
            System.out.println(result.substring(24, result.length() - 1));
            if (!StringUtils.isEmpty(result.substring(24, result.length() - 1))) {
                List<Map> maps = JSONArray.parseArray(result.substring(24, result.length() - 1), Map.class);
                List<Map> mapList = JSONArray.parseArray(JSON.parseObject(maps.get(0).get("listInfo").toString(), Map.class).get("content").toString(), Map.class);

                for (Map map : mapList) {
                    map.put("stock_code", map.get("companyCd"));
                    map.put("cname", map.get("companyName"));
                    map.put("overseas_flag", "0");
                    map.put("jys", "北京交易所");
                    map.put("shijian", map.get("publishDate"));
                    map.put("gonggao", map.get("disclosureTitle"));
                    map.put("pdf_url", "https://www.bse.cn" + map.get("destFilePath"));
                    if (map.get("disclosureTitle").toString().contains("(") && map.get("disclosureTitle").toString().contains(")")) {
                        map.put("gong_gao_type", map.get("disclosureTitle").toString().substring(map.get("disclosureTitle").toString().indexOf("（") + 1, map.get("disclosureTitle").toString().indexOf("）")));
                    }
                    System.out.println("保存数据");
                    crawlerAllMapper.insert(map);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            //释放连接
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
//                httpClient.close();
            }
        }
    }

    private void crawlerHsProps(Date date) throws IOException, ParseException {
        // 网址
        List<String> urlList = new ArrayList<>();
//        List<Date> dateList = DateUtil.findDates(DateUtil.lastedYearDate(new Date()), new Date());
//        for (Date date : dateList) {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        Document doc = getDocument(sdf.format(date), "http://eid.csrc.gov.cn/ipo/1010/index_f.html");
        if (doc.getElementsByClass("g-ul") == null || doc.getElementsByClass("g-ul").last() == null || doc.getElementsByClass("g-ul").last().children().last().select("li") == null
                || doc.getElementsByClass("g-ul").last().children().last().select("li").select("b") == null) {
            System.out.println("没数据！！");
        } else {

            String pageNum = doc.getElementsByClass("g-ul").last().children().last().select("li").select("b").text();

            // 上海
            for (int i = 1; i <= Integer.parseInt(pageNum); i++) {
                String url = "http://eid.csrc.gov.cn/ipo/1010/index_" + i + "_f.html";
                urlList.add(url);
            }

            executeData(urlList, date, sdf);
            urlList.clear();

            doc = getDocument(sdf.format(date), "http://eid.csrc.gov.cn/ipo/1017/index_f.html");
            if (doc != null &&
                    doc.getElementsByClass("g-ul") != null &&
                    doc.getElementsByClass("g-ul").last() != null &&
                    doc.getElementsByClass("g-ul").last().children() != null &&
                    doc.getElementsByClass("g-ul").last().children().last() != null &&
                    doc.getElementsByClass("g-ul").last().children().last().select("li") != null &&
                    doc.getElementsByClass("g-ul").last().children().last().select("li").select("b") != null &&
                    doc.getElementsByClass("g-ul").last().children().last().select("li").select("b").text() != null) {
                pageNum = doc.getElementsByClass("g-ul").last().children().last().select("li").select("b").text();
                // 深圳
                for (int i = 1; i <= Integer.parseInt(pageNum); i++) {
                    String url = "http://eid.csrc.gov.cn/ipo/1017/index_" + i + "_f.html";
                    urlList.add(url);
                }

                executeData(urlList, date, sdf);

            }
            urlList.clear();

        }
//        }
        System.out.println("爬取结束！");
    }

    private Document getDocument(String date, String url) throws IOException {
        System.out.println(url);
        Document document = Jsoup.connect(url)
                .data("startDate", date)
                .data("endDate", date)
                .userAgent("Mozilla")
                .cookie("auth", "token")
                .timeout(300000)
                .post();
        return document;
    }


    private void executeData(List<String> urlList, Date date, SimpleDateFormat sdf) throws IOException {
        for (String url : urlList) {
            System.out.println(sdf.format(date) + ":               " + url);
            Document document = getDocument(sdf.format(date), url);
            Elements elements = document.getElementsByClass("m-table2 m-table2-0");
            Element el = elements.first();
            List<Map<String, String>> tableColumnData = getTableColumnData(el, "1");
            for (Map<String, String> tableColumnDatum : tableColumnData) {
                crawlerAllMapper.insert(tableColumnDatum);
                System.out.println("保存数据");
            }
        }
    }

    /**
     * @param table
     * @param rowSelectRange
     * @return
     */
    public List<Map<String, String>> getTableColumnData(Element table, String rowSelectRange) {
        List<Map<String, String>> companyList = new ArrayList<>();

        Elements trs = table.select("tr");
        if (!StringUtils.isEmpty(rowSelectRange)) {
            String[] deleteRows = rowSelectRange.split(",");
            int offsetIndex = 0;
            for (int i = deleteRows.length - 1; i >= 0; i--) {
                int index = Integer.parseInt(deleteRows[i]);
                if (index < 0) {
                    index = Math.abs(index);
                    index = trs.size() - (index - offsetIndex);
                    trs.remove(index);
                    offsetIndex++;
                } else {
                    trs.remove(index - 1);
                }
            }
        }
        for (Element tr : trs) {
            Elements tds = tr.select("td");
            // th 和 td 混合的情况下，取子元素
            if (tr.select("th").size() > 0) {
                tds = tr.children();
            }
            int index = 0;
            Map<String, String> map = new HashMap<>();
            String urlPDF = String.valueOf(tr.attributes().get("onclick"));
            if (urlPDF.contains("(") && urlPDF.contains(")") && urlPDF.contains(".pdf") && urlPDF.contains("http")) {
                map.put("pdf_url", urlPDF.substring(urlPDF.indexOf("(") + 2, urlPDF.indexOf(".pdf") + 4));
            } else {
                map.put("pdf_url", null);
            }
            map.put("overseas_flag", "0");

            for (Element td : tds) {
                switch (index++) {
                    case 0:
                        if (StringUtils.isEmpty(td.text().trim())) continue;
                        map.put("cname", td.text().trim());
                        break;
                    case 1:
                        if (StringUtils.isEmpty(td.text().trim())) continue;
                        map.put("gong_gao_type", td.text().trim());
                        break;
                    case 2:
                        if (StringUtils.isEmpty(td.text().trim())) continue;
                        map.put("bankuai", td.text().trim());
                        break;
                    case 3:
                        if (StringUtils.isEmpty(td.text().trim())) continue;
                        map.put("jigou", td.text().trim());
                        break;
                    case 4:
                        if (StringUtils.isEmpty(td.text().trim())) continue;
                        map.put("shijian", td.text().trim());
                        break;
                    case 5:
                        if (StringUtils.isEmpty(td.text().trim())) continue;
                        map.put("gonggao", td.text().trim());
                        break;
                }
            }
            map.put("jys", map.get("bankuai").contains("深") ? "深圳交易所" : "上海交易所");
            companyList.add(map);
        }
        return companyList;
    }


}
